{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "056672d1",
   "metadata": {},
   "source": [
    "# OpenAI强化学习实战第5课书面作业\n",
    "学号：114488\n",
    "\n",
    "**作业内容：**  \n",
    "使用DQN方法训练Atari的一个游戏。"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d82a7bef",
   "metadata": {},
   "source": [
    "## 作业实现\n",
    "下面针对Atari中的Breakout游戏，使用DQN方法进行强化学习训练。\n",
    "\n",
    "过程中使用到了keras-rl2的库，简化了对于gym的操作。  \n",
    "同时Atari游戏的ROM在gym[atari]库中缺省是不带的，需要额外下载。ROM下载后通过命令\"ale-import-roms\"将ROM装载到系统中，注意该命令是区分虚拟空间的。\n",
    "\n",
    "这里采用的环境是“BreakoutDeterministic-v4”,在这个环境下直接输出的观察项就是游戏画面，采用CNN模型来实现，输入时，将原来RGB的图片统一到$84\\times 84\\times 1$的灰度图片，同时将连续4张图片（WINDOW_LENGTH=4）叠加作为模型输入。在学习过程中，采用$\\epsilon$贪婪策略。  \n",
    "整体实现源代码如下："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d07eb583",
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import numpy as np\n",
    "import gym\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "\n",
    "from tensorflow.keras.layers import Dense, Activation, Flatten, Convolution2D, Permute\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "import tensorflow.keras.backend as K\n",
    "\n",
    "from rl.agents.dqn import DQNAgent\n",
    "from rl.policy import LinearAnnealedPolicy, BoltzmannQPolicy, EpsGreedyQPolicy\n",
    "from rl.memory import SequentialMemory\n",
    "from rl.core import Processor\n",
    "from rl.callbacks import FileLogger, ModelIntervalCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "22ac439a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " permute (Permute)           (None, 84, 84, 4)         0         \n",
      "                                                                 \n",
      " conv2d (Conv2D)             (None, 20, 20, 32)        8224      \n",
      "                                                                 \n",
      " activation (Activation)     (None, 20, 20, 32)        0         \n",
      "                                                                 \n",
      " conv2d_1 (Conv2D)           (None, 9, 9, 64)          32832     \n",
      "                                                                 \n",
      " activation_1 (Activation)   (None, 9, 9, 64)          0         \n",
      "                                                                 \n",
      " conv2d_2 (Conv2D)           (None, 7, 7, 64)          36928     \n",
      "                                                                 \n",
      " activation_2 (Activation)   (None, 7, 7, 64)          0         \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 3136)              0         \n",
      "                                                                 \n",
      " dense (Dense)               (None, 512)               1606144   \n",
      "                                                                 \n",
      " activation_3 (Activation)   (None, 512)               0         \n",
      "                                                                 \n",
      " dense_1 (Dense)             (None, 4)                 2052      \n",
      "                                                                 \n",
      " activation_4 (Activation)   (None, 4)                 0         \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 1,686,180\n",
      "Trainable params: 1,686,180\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "INPUT_SHAPE = (84, 84)\n",
    "WINDOW_LENGTH = 4\n",
    "\n",
    "\n",
    "class AtariProcessor(Processor):\n",
    "    def process_observation(self, observation):\n",
    "        assert observation.ndim == 3  # (height, width, channel)\n",
    "        img = Image.fromarray(observation)\n",
    "        img = img.resize(INPUT_SHAPE).convert('L')  # resize and convert to grayscale\n",
    "        processed_observation = np.array(img)\n",
    "        assert processed_observation.shape == INPUT_SHAPE\n",
    "        return processed_observation.astype('uint8')  # saves storage in experience memory\n",
    "\n",
    "    def process_state_batch(self, batch):\n",
    "        processed_batch = batch.astype('float32') / 255.\n",
    "        return processed_batch\n",
    "\n",
    "    def process_reward(self, reward):\n",
    "        return np.clip(reward, -1., 1.)\n",
    "\n",
    "# Get the environment and extract the number of actions.\n",
    "env = gym.make('BreakoutDeterministic-v4')\n",
    "np.random.seed(123)\n",
    "env.seed(123)\n",
    "nb_actions = env.action_space.n\n",
    "\n",
    "# Next, we build our model. We use the same model that was described by Mnih et al. (2015).\n",
    "input_shape = (WINDOW_LENGTH,) + INPUT_SHAPE\n",
    "model = Sequential()\n",
    "\n",
    "# (width, height, channels)\n",
    "model.add(Permute((2, 3, 1), input_shape=input_shape))\n",
    "\n",
    "model.add(Convolution2D(32, (8, 8), strides=(4, 4)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Convolution2D(64, (4, 4), strides=(2, 2)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Convolution2D(64, (3, 3), strides=(1, 1)))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(512))\n",
    "model.add(Activation('relu'))\n",
    "model.add(Dense(nb_actions))\n",
    "model.add(Activation('linear'))\n",
    "print(model.summary())\n",
    "\n",
    "memory = SequentialMemory(limit=1000000, window_length=WINDOW_LENGTH)\n",
    "processor = AtariProcessor()\n",
    "policy = LinearAnnealedPolicy(EpsGreedyQPolicy(), attr='eps', value_max=1., value_min=.1, value_test=.05,\n",
    "                              nb_steps=1000000)\n",
    "dqn = DQNAgent(model=model, nb_actions=nb_actions, policy=policy, memory=memory,\n",
    "               processor=processor, nb_steps_warmup=50000, gamma=.99, target_model_update=10000,\n",
    "               train_interval=4, delta_clip=1.)\n",
    "dqn.compile(Adam(learning_rate=.00025), metrics=['mae'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9be484ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training for 180000 steps ...\n",
      "Interval 1 (0 steps performed)\n",
      "   40/10000 [..............................] - ETA: 26s - reward: 0.0000e+00"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\13917\\anaconda3\\lib\\site-packages\\keras\\engine\\training_v1.py:2079: UserWarning: `Model.state_updates` will be removed in a future version. This property should not be used in TensorFlow 2.0, as `updates` are applied automatically.\n",
      "  updates=self.state_updates,\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 26s 3ms/step - reward: 0.0067\n",
      "54 episodes - episode_reward: 1.241 [0.000, 5.000] - lives: 2.958\n",
      "\n",
      "Interval 2 (10000 steps performed)\n",
      "10000/10000 [==============================] - 26s 3ms/step - reward: 0.0059\n",
      "56 episodes - episode_reward: 1.036 [0.000, 4.000] - lives: 2.978\n",
      "\n",
      "Interval 3 (20000 steps performed)\n",
      "10000/10000 [==============================] - 26s 3ms/step - reward: 0.0060\n",
      "56 episodes - episode_reward: 1.089 [0.000, 4.000] - lives: 2.930\n",
      "\n",
      "Interval 4 (30000 steps performed)\n",
      "10000/10000 [==============================] - 26s 3ms/step - reward: 0.0055\n",
      "58 episodes - episode_reward: 0.948 [0.000, 5.000] - lives: 2.949\n",
      "\n",
      "Interval 5 (40000 steps performed)\n",
      "10000/10000 [==============================] - 26s 3ms/step - reward: 0.0084\n",
      "50 episodes - episode_reward: 1.640 [0.000, 5.000] - lives: 3.061\n",
      "\n",
      "Interval 6 (50000 steps performed)\n",
      "10000/10000 [==============================] - 248s 25ms/step - reward: 0.0055\n",
      "57 episodes - episode_reward: 1.000 [0.000, 4.000] - loss: 0.002 - mae: 0.057 - mean_q: 0.080 - mean_eps: 0.951 - lives: 2.924\n",
      "\n",
      "Interval 7 (60000 steps performed)\n",
      "10000/10000 [==============================] - 250s 25ms/step - reward: 0.0054\n",
      "58 episodes - episode_reward: 0.897 [0.000, 4.000] - loss: 0.001 - mae: 0.064 - mean_q: 0.087 - mean_eps: 0.942 - lives: 2.961\n",
      "\n",
      "Interval 8 (70000 steps performed)\n",
      "10000/10000 [==============================] - 250s 25ms/step - reward: 0.0064\n",
      "56 episodes - episode_reward: 1.179 [0.000, 5.000] - loss: 0.001 - mae: 0.069 - mean_q: 0.093 - mean_eps: 0.933 - lives: 2.905\n",
      "\n",
      "Interval 9 (80000 steps performed)\n",
      "10000/10000 [==============================] - 249s 25ms/step - reward: 0.0074\n",
      "52 episodes - episode_reward: 1.423 [0.000, 6.000] - loss: 0.001 - mae: 0.074 - mean_q: 0.100 - mean_eps: 0.924 - lives: 2.961\n",
      "\n",
      "Interval 10 (90000 steps performed)\n",
      "10000/10000 [==============================] - 249s 25ms/step - reward: 0.0072\n",
      "52 episodes - episode_reward: 1.385 [0.000, 5.000] - loss: 0.001 - mae: 0.081 - mean_q: 0.109 - mean_eps: 0.915 - lives: 2.957\n",
      "\n",
      "Interval 11 (100000 steps performed)\n",
      "10000/10000 [==============================] - 250s 25ms/step - reward: 0.0062\n",
      "56 episodes - episode_reward: 1.107 [0.000, 4.000] - loss: 0.001 - mae: 0.086 - mean_q: 0.116 - mean_eps: 0.906 - lives: 2.987\n",
      "\n",
      "Interval 12 (110000 steps performed)\n",
      "10000/10000 [==============================] - 250s 25ms/step - reward: 0.0074\n",
      "53 episodes - episode_reward: 1.396 [0.000, 7.000] - loss: 0.001 - mae: 0.098 - mean_q: 0.131 - mean_eps: 0.897 - lives: 2.934\n",
      "\n",
      "Interval 13 (120000 steps performed)\n",
      "10000/10000 [==============================] - 249s 25ms/step - reward: 0.0066\n",
      "54 episodes - episode_reward: 1.185 [0.000, 5.000] - loss: 0.001 - mae: 0.101 - mean_q: 0.135 - mean_eps: 0.888 - lives: 2.944\n",
      "\n",
      "Interval 14 (130000 steps performed)\n",
      "10000/10000 [==============================] - 252s 25ms/step - reward: 0.0068\n",
      "55 episodes - episode_reward: 1.273 [0.000, 5.000] - loss: 0.001 - mae: 0.109 - mean_q: 0.145 - mean_eps: 0.879 - lives: 2.966\n",
      "\n",
      "Interval 15 (140000 steps performed)\n",
      "10000/10000 [==============================] - 252s 25ms/step - reward: 0.0069\n",
      "54 episodes - episode_reward: 1.278 [0.000, 4.000] - loss: 0.001 - mae: 0.120 - mean_q: 0.160 - mean_eps: 0.870 - lives: 3.038\n",
      "\n",
      "Interval 16 (150000 steps performed)\n",
      "10000/10000 [==============================] - 253s 25ms/step - reward: 0.0075\n",
      "51 episodes - episode_reward: 1.431 [0.000, 4.000] - loss: 0.001 - mae: 0.129 - mean_q: 0.173 - mean_eps: 0.861 - lives: 2.945\n",
      "\n",
      "Interval 17 (160000 steps performed)\n",
      "10000/10000 [==============================] - 255s 25ms/step - reward: 0.0074\n",
      "54 episodes - episode_reward: 1.370 [0.000, 6.000] - loss: 0.001 - mae: 0.136 - mean_q: 0.182 - mean_eps: 0.852 - lives: 2.917\n",
      "\n",
      "Interval 18 (170000 steps performed)\n",
      "10000/10000 [==============================] - 256s 26ms/step - reward: 0.0062\n",
      "done, took 3395.355 seconds\n",
      "Testing for 10 episodes ...\n",
      "Episode 1: reward: 2.000, steps: 100000\n",
      "Episode 2: reward: 2.000, steps: 100000\n",
      "Episode 3: reward: 2.000, steps: 100000\n",
      "Episode 4: reward: 2.000, steps: 100000\n",
      "Episode 5: reward: 2.000, steps: 100000\n",
      "Episode 6: reward: 2.000, steps: 100000\n",
      "Episode 7: reward: 2.000, steps: 100000\n",
      "Episode 8: reward: 2.000, steps: 100000\n",
      "Episode 9: reward: 2.000, steps: 100000\n",
      "Episode 10: reward: 2.000, steps: 100000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2192a4514f0>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights_filename = 'dqn_{}_weights.h5f'.format('BreakoutDeterministic-v4')\n",
    "checkpoint_weights_filename = 'dqn_' + 'BreakoutDeterministic-v4' + '_weights_{step}.h5f'\n",
    "log_filename = 'dqn_{}_log.json'.format('BreakoutDeterministic-v4')\n",
    "callbacks = [ModelIntervalCheckpoint(checkpoint_weights_filename, interval=250000)]\n",
    "callbacks += [FileLogger(log_filename, interval=100)]\n",
    "dqn.fit(env, callbacks=callbacks, nb_steps=180000, log_interval=10000)\n",
    "\n",
    "# After training is done, we save the final weights one more time.\n",
    "dqn.save_weights(weights_filename, overwrite=True)\n",
    "\n",
    "# Finally, evaluate our algorithm for 10 episodes.\n",
    "dqn.test(env, nb_episodes=10, visualize=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2bfbe487",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Testing for 10 episodes ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\13917\\anaconda3\\lib\\site-packages\\gym\\envs\\atari\\environment.py:255: UserWarning: \u001b[33mWARN: We strongly suggest supplying `render_mode` when constructing your environment, e.g., gym.make(ID, render_mode='human'). Using `render_mode` provides access to proper scaling, audio support, and proper framerates.\u001b[0m\n",
      "  logger.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 1: reward: 2.000, steps: 100000\n",
      "Episode 2: reward: 2.000, steps: 100000\n",
      "Episode 3: reward: 2.000, steps: 100000\n",
      "Episode 4: reward: 2.000, steps: 100000\n",
      "Episode 5: reward: 2.000, steps: 100000\n",
      "Episode 6: reward: 2.000, steps: 100000\n",
      "Episode 7: reward: 2.000, steps: 100000\n",
      "Episode 8: reward: 2.000, steps: 100000\n",
      "Episode 9: reward: 2.000, steps: 100000\n",
      "Episode 10: reward: 2.000, steps: 100000\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2191a467df0>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weights_filename = 'dqn_{}_weights.h5f'.format('BreakoutDeterministic-v4')\n",
    "dqn.load_weights(weights_filename)\n",
    "dqn.test(env, nb_episodes=10, visualize=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
